<!doctype html>
<html class="no-js" lang="en" data-content_root="./">
  <head><meta charset="utf-8"/>
    <meta name="viewport" content="width=device-width,initial-scale=1"/>
    <meta name="color-scheme" content="light dark"><meta name="viewport" content="width=device-width, initial-scale=1" />
<link rel="index" title="Index" href="genindex.html" /><link rel="search" title="Search" href="search.html" /><link rel="next" title="Components and Features" href="components.html" /><link rel="prev" title="Demos" href="demos.html" />

    <!-- Generated with Sphinx 7.3.7 and Furo 2024.05.06 -->
        <title>UI Guide - 🔥LIT 1.0 documentation</title>
      <link rel="stylesheet" type="text/css" href="_static/pygments.css?v=a746c00c" />
    <link rel="stylesheet" type="text/css" href="_static/styles/furo.css?v=387cc868" />
    <link rel="stylesheet" type="text/css" href="_static/styles/furo-extensions.css?v=36a5483c" />
    <link rel="stylesheet" type="text/css" href="_static/furo_custom.css?v=6173ce67" />
    
    


<style>
  body {
    --color-code-background: #f8f8f8;
  --color-code-foreground: black;
  
  }
  @media not print {
    body[data-theme="dark"] {
      --color-code-background: #202020;
  --color-code-foreground: #d0d0d0;
  
    }
    @media (prefers-color-scheme: dark) {
      body:not([data-theme="light"]) {
        --color-code-background: #202020;
  --color-code-foreground: #d0d0d0;
  
      }
    }
  }
</style></head>
  <body>
    
    <script>
      document.body.dataset.theme = localStorage.getItem("theme") || "auto";
    </script>
    

<svg xmlns="http://www.w3.org/2000/svg" style="display: none;">
  <symbol id="svg-toc" viewBox="0 0 24 24">
    <title>Contents</title>
    <svg stroke="currentColor" fill="currentColor" stroke-width="0" viewBox="0 0 1024 1024">
      <path d="M408 442h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8zm-8 204c0 4.4 3.6 8 8 8h480c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8H408c-4.4 0-8 3.6-8 8v56zm504-486H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zm0 632H120c-4.4 0-8 3.6-8 8v56c0 4.4 3.6 8 8 8h784c4.4 0 8-3.6 8-8v-56c0-4.4-3.6-8-8-8zM115.4 518.9L271.7 642c5.8 4.6 14.4.5 14.4-6.9V388.9c0-7.4-8.5-11.5-14.4-6.9L115.4 505.1a8.74 8.74 0 0 0 0 13.8z"/>
    </svg>
  </symbol>
  <symbol id="svg-menu" viewBox="0 0 24 24">
    <title>Menu</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-menu">
      <line x1="3" y1="12" x2="21" y2="12"></line>
      <line x1="3" y1="6" x2="21" y2="6"></line>
      <line x1="3" y1="18" x2="21" y2="18"></line>
    </svg>
  </symbol>
  <symbol id="svg-arrow-right" viewBox="0 0 24 24">
    <title>Expand</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="2" stroke-linecap="round" stroke-linejoin="round" class="feather-chevron-right">
      <polyline points="9 18 15 12 9 6"></polyline>
    </svg>
  </symbol>
  <symbol id="svg-sun" viewBox="0 0 24 24">
    <title>Light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="feather-sun">
      <circle cx="12" cy="12" r="5"></circle>
      <line x1="12" y1="1" x2="12" y2="3"></line>
      <line x1="12" y1="21" x2="12" y2="23"></line>
      <line x1="4.22" y1="4.22" x2="5.64" y2="5.64"></line>
      <line x1="18.36" y1="18.36" x2="19.78" y2="19.78"></line>
      <line x1="1" y1="12" x2="3" y2="12"></line>
      <line x1="21" y1="12" x2="23" y2="12"></line>
      <line x1="4.22" y1="19.78" x2="5.64" y2="18.36"></line>
      <line x1="18.36" y1="5.64" x2="19.78" y2="4.22"></line>
    </svg>
  </symbol>
  <symbol id="svg-moon" viewBox="0 0 24 24">
    <title>Dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-moon">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M12 3c.132 0 .263 0 .393 0a7.5 7.5 0 0 0 7.92 12.446a9 9 0 1 1 -8.313 -12.454z" />
    </svg>
  </symbol>
  <symbol id="svg-sun-half" viewBox="0 0 24 24">
    <title>Auto light/dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-shadow">
      <path stroke="none" d="M0 0h24v24H0z" fill="none"/>
      <circle cx="12" cy="12" r="9" />
      <path d="M13 12h5" />
      <path d="M13 15h4" />
      <path d="M13 18h1" />
      <path d="M13 9h4" />
      <path d="M13 6h1" />
    </svg>
  </symbol>
  <symbol id="svg-sun-with-moon" viewBox="0 0 24 24">
    <title>Auto light/dark, in light mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path style="opacity: 50%" d="M 5.411 14.504 C 5.471 14.504 5.532 14.504 5.591 14.504 C 3.639 16.319 4.383 19.569 6.931 20.352 C 7.693 20.586 8.512 20.551 9.25 20.252 C 8.023 23.207 4.056 23.725 2.11 21.184 C 0.166 18.642 1.702 14.949 4.874 14.536 C 5.051 14.512 5.231 14.5 5.411 14.5 L 5.411 14.504 Z"/>
      <line x1="14.5" y1="3.25" x2="14.5" y2="1.25"/>
      <line x1="14.5" y1="15.85" x2="14.5" y2="17.85"/>
      <line x1="10.044" y1="5.094" x2="8.63" y2="3.68"/>
      <line x1="19" y1="14.05" x2="20.414" y2="15.464"/>
      <line x1="8.2" y1="9.55" x2="6.2" y2="9.55"/>
      <line x1="20.8" y1="9.55" x2="22.8" y2="9.55"/>
      <line x1="10.044" y1="14.006" x2="8.63" y2="15.42"/>
      <line x1="19" y1="5.05" x2="20.414" y2="3.636"/>
      <circle cx="14.5" cy="9.55" r="3.6"/>
    </svg>
  </symbol>
  <symbol id="svg-moon-with-sun" viewBox="0 0 24 24">
    <title>Auto light/dark, in dark mode</title>
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round"
      class="icon-custom-derived-from-feather-sun-and-tabler-moon">
      <path d="M 8.282 7.007 C 8.385 7.007 8.494 7.007 8.595 7.007 C 5.18 10.184 6.481 15.869 10.942 17.24 C 12.275 17.648 13.706 17.589 15 17.066 C 12.851 22.236 5.91 23.143 2.505 18.696 C -0.897 14.249 1.791 7.786 7.342 7.063 C 7.652 7.021 7.965 7 8.282 7 L 8.282 7.007 Z"/>
      <line style="opacity: 50%" x1="18" y1="3.705" x2="18" y2="2.5"/>
      <line style="opacity: 50%" x1="18" y1="11.295" x2="18" y2="12.5"/>
      <line style="opacity: 50%" x1="15.316" y1="4.816" x2="14.464" y2="3.964"/>
      <line style="opacity: 50%" x1="20.711" y1="10.212" x2="21.563" y2="11.063"/>
      <line style="opacity: 50%" x1="14.205" y1="7.5" x2="13.001" y2="7.5"/>
      <line style="opacity: 50%" x1="21.795" y1="7.5" x2="23" y2="7.5"/>
      <line style="opacity: 50%" x1="15.316" y1="10.184" x2="14.464" y2="11.036"/>
      <line style="opacity: 50%" x1="20.711" y1="4.789" x2="21.563" y2="3.937"/>
      <circle style="opacity: 50%" cx="18" cy="7.5" r="2.169"/>
    </svg>
  </symbol>
  <symbol id="svg-pencil" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-pencil-code">
      <path d="M4 20h4l10.5 -10.5a2.828 2.828 0 1 0 -4 -4l-10.5 10.5v4" />
      <path d="M13.5 6.5l4 4" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" /
    </svg>
  </symbol>
  <symbol id="svg-eye" viewBox="0 0 24 24">
    <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor"
      stroke-width="1" stroke-linecap="round" stroke-linejoin="round" class="icon-tabler-eye-code">
      <path stroke="none" d="M0 0h24v24H0z" fill="none" />
      <path d="M10 12a2 2 0 1 0 4 0a2 2 0 0 0 -4 0" />
      <path
        d="M11.11 17.958c-3.209 -.307 -5.91 -2.293 -8.11 -5.958c2.4 -4 5.4 -6 9 -6c3.6 0 6.6 2 9 6c-.21 .352 -.427 .688 -.647 1.008" />
      <path d="M20 21l2 -2l-2 -2" />
      <path d="M17 17l-2 2l2 2" />
    </svg>
  </symbol>
</svg>

<input type="checkbox" class="sidebar-toggle" name="__navigation" id="__navigation">
<input type="checkbox" class="sidebar-toggle" name="__toc" id="__toc">
<label class="overlay sidebar-overlay" for="__navigation">
  <div class="visually-hidden">Hide navigation sidebar</div>
</label>
<label class="overlay toc-overlay" for="__toc">
  <div class="visually-hidden">Hide table of contents sidebar</div>
</label>

<a class="skip-to-content muted-link" href="#furo-main-content">Skip to content</a>



<div class="page">
  <header class="mobile-header">
    <div class="header-left">
      <label class="nav-overlay-icon" for="__navigation">
        <div class="visually-hidden">Toggle site navigation sidebar</div>
        <i class="icon"><svg><use href="#svg-menu"></use></svg></i>
      </label>
    </div>
    <div class="header-center">
      <a href="index.html"><div class="brand">🔥LIT 1.0 documentation</div></a>
    </div>
    <div class="header-right">
      <div class="theme-toggle-container theme-toggle-header">
        <button class="theme-toggle">
          <div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
          <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
          <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
          <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
          <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
        </button>
      </div>
      <label class="toc-overlay-icon toc-header-icon" for="__toc">
        <div class="visually-hidden">Toggle table of contents sidebar</div>
        <i class="icon"><svg><use href="#svg-toc"></use></svg></i>
      </label>
    </div>
  </header>
  <aside class="sidebar-drawer">
    <div class="sidebar-container">
      
      <div class="sidebar-sticky"><a class="sidebar-brand" href="index.html">
  
  
  <span class="sidebar-brand-text">🔥LIT 1.0 documentation</span>
  
</a><form class="sidebar-search-container" method="get" action="search.html" role="search">
  <input class="sidebar-search" placeholder="Search" name="q" aria-label="Search">
  <input type="hidden" name="check_keywords" value="yes">
  <input type="hidden" name="area" value="default">
</form>
<div id="searchbox"></div><div class="sidebar-scroll"><div class="sidebar-tree">
  <ul class="current">
<li class="toctree-l1"><a class="reference external" href="https://pair-code.github.io/lit/">   Main Site</a></li>
<li class="toctree-l1"><a class="reference internal" href="getting_started.html">   Getting Started</a></li>
<li class="toctree-l1"><a class="reference internal" href="demos.html">   Examples</a></li>
<li class="toctree-l1 current current-page"><a class="current reference internal" href="#">   UI Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="components.html">   Components &amp; Features</a></li>
<li class="toctree-l1"><a class="reference internal" href="api.html">   Python API</a></li>
<li class="toctree-l1"><a class="reference internal" href="frontend_development.html">   Frontend Development</a></li>
<li class="toctree-l1"><a class="reference internal" href="docker.html">   Running in Docker</a></li>
<li class="toctree-l1"><a class="reference internal" href="glossary.html">   Glossary</a></li>
<li class="toctree-l1"><a class="reference internal" href="faq.html">   FAQ</a></li>
<li class="toctree-l1"><a class="reference external" href="https://github.com/pair-code/lit">   GitHub</a></li>
</ul>

</div>
</div>

      </div>
      
    </div>
  </aside>
  <div class="main">
    <div class="content">
      <div class="article-container">
        <a href="#" class="back-to-top muted-link">
          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24">
            <path d="M13 20h-2V8l-5.5 5.5-1.42-1.42L12 4.16l7.92 7.92-1.42 1.42L13 8v12z"></path>
          </svg>
          <span>Back to top</span>
        </a>
        <div class="content-icon-container">
          <div class="view-this-page">
  <a class="muted-link" href="_sources/ui_guide.md.txt" title="View this page">
    <svg><use href="#svg-eye"></use></svg>
    <span class="visually-hidden">View this page</span>
  </a>
</div>
<div class="theme-toggle-container theme-toggle-content">
            <button class="theme-toggle">
              <div class="visually-hidden">Toggle Light / Dark / Auto color theme</div>
              <svg class="theme-icon-when-auto-light"><use href="#svg-sun-with-moon"></use></svg>
              <svg class="theme-icon-when-auto-dark"><use href="#svg-moon-with-sun"></use></svg>
              <svg class="theme-icon-when-dark"><use href="#svg-moon"></use></svg>
              <svg class="theme-icon-when-light"><use href="#svg-sun"></use></svg>
            </button>
          </div>
          <label class="toc-overlay-icon toc-content-icon" for="__toc">
            <div class="visually-hidden">Toggle table of contents sidebar</div>
            <i class="icon"><svg><use href="#svg-toc"></use></svg></i>
          </label>
        </div>
        <article role="main" id="furo-main-content">
          <section class="tex2jax_ignore mathjax_ignore" id="ui-guide">
<h1>UI Guide<a class="headerlink" href="#ui-guide" title="Link to this heading">¶</a></h1>
<!--* freshness: { owner: 'lit-dev' reviewed: '2024-06-24' } *-->
<p>This is a user guide for the Learning Interpretability Tool (LIT) UI.</p>
<p>For a quick video tour of LIT, check out this
<a class="reference external" href="https://www.youtube.com/watch?v=CuRI_VK83dU">video</a>.</p>
<!-- [TOC] placeholder - DO NOT REMOVE -->
<section id="general-layout">
<h2>General Layout<a class="headerlink" href="#general-layout" title="Link to this heading">¶</a></h2>
<p>LIT lives inside a single page web application, comprised of multiple toolbars
and a main section consisting of individual modules. Modules will automatically
display if they are applicable to the current model and dataset; for example,
the module that shows classification results will only show if the model returns
<code class="docutils literal notranslate"><span class="pre">MulticlassPreds</span></code>.</p>
<p><img alt="LIT overall UI" src="_images/lit-ui.png" /></p>
<p>LIT’s layout consist of as many as three sections, described in the API docs for
<a class="reference internal" href="api.html#customizing-the-layout"><span class="std std-ref">custom layouts</span></a>. When the layout provides more than one
major content section, they are separated by draggable dividers that are built
into LIT’s toolbars (for allocating vertical space) or in the space between
sections and modules (for allocating horizontal space). Any section may include
multiple tabs, where each tab contains a collection of modules. LIT’s
<a class="reference internal" href="frontend_development.html#layout"><span class="std std-ref">pre-configured layouts</span></a> group modules into
tabs based on analytical task (e.g., metrics analysis vs. input salience
visualization vs. counterfactual example generation), but you can adopt whatever
organizational scheme you desire in your custom layouts.</p>
<section id="layout-options">
<h3>Layout Options<a class="headerlink" href="#layout-options" title="Link to this heading">¶</a></h3>
<!--
  TODO: Add an image of the 3 LIT layouts side by side.
-->
<p>LIT provides three pre-configured layouts:</p>
<ul class="simple">
<li><p><code class="docutils literal notranslate"><span class="pre">simple</span></code>: A minimalist layout with the examples on top (either individually
(selected by default) or in a table) and predictions on the bottom;</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">default</span></code>: The original LIT layout with a single group of modules on top for
exploring and selecting data, and a collection of tabs supporting different
analytical tasks on the bottom; and</p></li>
<li><p><code class="docutils literal notranslate"><span class="pre">three_panel</span></code>: A three-panel layout that puts exploratory data
visualizations at full-page height on the left, tools for inspecting and
manipulating examples and their associated predictions in the upper right,
and a collection of tabs supporting different analytical tasks in the lower
right. Note that this was introduced in v1.0 as an experimental feature,
your feedback is appreciated.</p></li>
</ul>
</section>
</section>
<section id="datapoint-selections">
<h2>Datapoint Selections<a class="headerlink" href="#datapoint-selections" title="Link to this heading">¶</a></h2>
<p>LIT displays a loaded dataset and its model results across the set of selected
models. Users can dive into detailed results by selecting datapoints from the
dataset.</p>
<p><img alt="LIT datapoint selection" src="_images/lit-datapoint-selection.png" /></p>
<p>LIT provides two levels of precision for selections. The first is the current
selection, which consists of one or more datapoints that are selected through
one of the interactive modules (such as the <em>Data Table</em>, <em>Embeddings</em>,
<em>Scalars</em>, or <em>Confusion Matrix</em> module). When a set of datapoints is selected
in a module, this selection is reflected across all other modules, along with
the selection toolbar. For example, the <em>Metrics</em> module shows model metrics not
just across the entire dataset, but also for the current selection of
datapoints.</p>
<p>The second is the primary selection. This is a single datapoint within the
current selection that is being explored in more detail in modules that focus on
a single datapoint (such as the <em>Datapoint Editor</em> and <em>Salience Maps</em> modules).
If the current selection only consists of a single datapoint, then that
datapoint is also the primary selection. If the current selection consists of
multiple datapoints, the primary selection defaults to the first datapoint in
that selection but can be changed through the arrow controls in the selection
toolbar or by clicking another datapoint in the selection. The primary selection
is highlighted in a darker blue in the <em>Data Table</em> module and its ID is
displayed in the selection toolbar.</p>
<p>A selection of datapoints can be saved as a “slice” through the
<em><a class="reference internal" href="#slices">Slice Editor</a></em>. Saving a selection as a slice allows for easy
navigation back to that selection in the future. It also allows for comparison
of metrics across subsets of datapoints, as described in the
<em><a class="reference internal" href="#metrics-table">Metrics Module</a></em> section.</p>
</section>
<section id="toolbars">
<h2>Toolbars<a class="headerlink" href="#toolbars" title="Link to this heading">¶</a></h2>
<p>There are three toolbars provided in LIT. The top bar includes the selected
model(s) and dataset, a settings button, and URL sharing functionality. Below
that is the main toolbar with the menus and controls for navigation and
selection. At the bottom of the page is a status bar.</p>
<p><img alt="LIT toolbars" src="_images/lit-toolbars.png" /></p>
<section id="top-bar">
<h3>Top Bar<a class="headerlink" href="#top-bar" title="Link to this heading">¶</a></h3>
<section id="global-settings">
<h4>Global Settings<a class="headerlink" href="#global-settings" title="Link to this heading">¶</a></h4>
<p>The global settings dialog is accessible through the <strong>“Configure”</strong> button in
the top bar.</p>
<p>LIT can be launched with a set of models and datasets. The settings screen
allows users to select which models to analyze. Any number of models can be
analyzed together, assuming they are compatible in the input data format they
use (i.e. two different toxicity classifiers can be analyzed together for
comparison). Once a model or models is selected, you can then select from any
dataset compatible with those models.</p>
<p>The settings dialog also contains controls switching the layout of the tool.
This can help declutter the UI when analysis doesn’t require all of the
compatible modules that LIT contains.</p>
<p><img alt="LIT global settings" src="_images/lit-settings.png" /></p>
</section>
<section id="url-sharing">
<h4>URL Sharing<a class="headerlink" href="#url-sharing" title="Link to this heading">¶</a></h4>
<p>Much of the LIT app’s state — the loaded models and datasets, selected
datapoints, minimized and/or full-screen modules — is stored in URL
parameters. The <strong>“Copy Link”</strong> button in the top bar allows a user to share
their specific LIT view and setup with someone else. The URL can also be copied
manually from the address bar.</p>
<p>The base url that will be copied with the <strong>“Copy Link”</strong> button can be
configured by passing the <code class="docutils literal notranslate"><span class="pre">--canonical_url=&lt;url</span> <span class="pre">base&gt;</span></code> flag to the server.</p>
</section>
</section>
<section id="main-toolbar">
<h3>Main Toolbar<a class="headerlink" href="#main-toolbar" title="Link to this heading">¶</a></h3>
<p>The main toolbar is right below the top bar and contains a number of different
controls and information. The left side of the toolbar contains a set of menus
for quickly controlling datapoint selection and coloring. This includes the
following controls:</p>
<ul class="simple">
<li><p>The <strong>“Select datapoint”</strong> menu provides a drop-down of several options:</p>
<ul>
<li><p>the <strong>“Random”</strong> option selects a random datapoint,</p></li>
<li><p>the <strong>“All related”</strong> option adds any datapoints “related” to the
current selection. In LIT, “related” is defined as datapoints created
from some source datapoint (through manual editing or a datapoint
generator), or a source datapoint that a selected datapoint was created
from,</p></li>
<li><p>the <strong>“Parents”</strong> option adds the source datapoints that the selected
datapoints were created from,</p></li>
<li><p>the <strong>“Children”</strong> option adds the datapoints created from the selected
datapoints (through manual editing or a datapoint generator),</p></li>
<li><p>the <strong>Slices</strong> option allows quick selection of an already-created slice
of datapoints,</p></li>
<li><p>the <strong>“Clear selection”</strong> button deselects all selected datapoints.</p></li>
</ul>
</li>
<li><p>The <strong>“Color by”</strong> menu enables setting of the color of each datapoint in
the modules that visualize all datapoints (such as the <em>Embeddings</em> and
<em>Scalars</em> modules) by any number of datapoint features or model outputs on
those datapoints (such as coloring by some categorical input feature, or by
prediction error for a regression task).</p></li>
<li><p>The <strong>Slices</strong> menu allows adding/selecting/removing slices of datapoints.</p></li>
</ul>
<p>Next to the menus is a button for pinning/unpinning a datapoint. Pinning a
datapoint puts LIT into datapoint comparison mode, where two datapoints can be
compared against each other, across all applicable modules. This mode is
described in more detail <a class="reference internal" href="#comparing-datapoints">below</a>.</p>
<p>The right side of the toolbar displays how many datapoints are in the loaded
dataset and how many of those are currently selected. If only a single datapoint
is selected, the left and right arrow buttons in this toolbar allow cycling of
the selected datapoint through the loaded dataset. If the current selection is a
set of datapoints, then the left and right arrow buttons control which of those
datapoints is the primary selected datapoint, cycling through the datapoints in
the current selection. A <strong>“Select random”</strong> button allows selection of a random
datapoint, as opposed to the ordered cycling done through the left and right
arrows.The <strong>“Select all”</strong> and <strong>“Clear selection”</strong> buttons are also provided
to easily select all or none of the datapoints, respectively.</p>
</section>
<section id="status-bar">
<h3>Status Bar<a class="headerlink" href="#status-bar" title="Link to this heading">¶</a></h3>
<p>The status bar at the bottom of the tool contains a text area on the left side.
If the tool is currently waiting on the results of a call to the backend (such
as for running predictions or getting embeddings), this information will be
displayed in the status bar along with an indeterminant progress bar showing
that a result is pending. If a call to the backend fails, information about the
failure will be displayed in this area in red to call out the error, and that
information will persist in the status bar until the user clicks the <strong>“x”</strong>
button by the error to clear the status display. The full error log can also be
displayed by clicking the error icon in the message.</p>
</section>
</section>
<section id="comparing-models">
<h2>Comparing Models<a class="headerlink" href="#comparing-models" title="Link to this heading">¶</a></h2>
<p>By loading more than one model in the global settings controls, LIT can compare
multiple models. A subset of modules that show per-model information are then
duplicated to allow easy comparison across two models. Other modules, such the
<em>Embeddings</em> and <em>Metrics</em> modules are updated to show information from all
models.</p>
<p><img alt="LIT model comparison" src="_images/lit-model-compare.png" /></p>
</section>
<section id="comparing-datapoints">
<h2>Comparing Datapoints<a class="headerlink" href="#comparing-datapoints" title="Link to this heading">¶</a></h2>
<p>Pinning a datapoint, through either the toolbar button or controls in modules
(e.g., the pin icons in Data Table rows), puts LIT into <strong>datapoint comparison
mode</strong>. In this mode, the pinned datapoint is used as a reference to compare the
primary selection. The pinned datapoint is indicated by a pin icon in modules
that support datapoint comparison, such as the Data Table. Any changes to the
primary selection will update datapoint comparison visualizations in all
supporting modules.</p>
<p>As with model comparison, some modules may be duplicated, one showing the pinned
datapoint and one showing the primary selected datapoint.</p>
<p>This allows for easy comparison of model results on a datapoint to any generated
counterfactual datapoints, or any other datapoint from the loaded dataset.</p>
<p><img alt="LIT datapoint comparison" src="_images/lit-datapoint-compare.png" /></p>
</section>
<section id="slices">
<h2>Slices<a class="headerlink" href="#slices" title="Link to this heading">¶</a></h2>
<p>The <em>Slice Editor</em> allow users to create, edit, select, and delete slices. The
current selection can be saved as a slice by giving it a name and clicking
“Create slice”. The slice list allows you to select any of the previously-saved
slices. This includes the “Starred” slice that is described above in the
<a class="reference internal" href="#main-toolbar">Main Toolbar</a> section.</p>
<p>The feature checkboxes enable the user to facet the data by input feature when
creating a slice. In the screenshot below, we are creating a new slice named
“interesting”, and have selected the checkbox to facet by the “label” feature.
In this example, the “label” feature is a feature in the dataset that for each
datapoint describes which ground truth class it belongs to for some
classification task (either “0” or “1” for this binary classification example).
So, by creating a slice with this checkbox enabled, the tool will actually
create two slices: one named “interesting label:0” for datapoints with their
label set to 0, and one named “interesting label:1” for those with their label
set to “1”.</p>
<p><img alt="LIT slice controls" src="_images/lit-slices.png" /></p>
</section>
<section id="module-details">
<h2>Module Details<a class="headerlink" href="#module-details" title="Link to this heading">¶</a></h2>
<p>This section contains details on using and interacting with individual modules
that are built into LIT. Note that this list may not be complete and additional
modules can be created and used in LIT by clients.</p>
<p>All modules can be toggled to be shown full-screen through use of the
full-screen button in the top-right of each module.</p>
<section id="embedding-projector">
<h3>Embedding Projector<a class="headerlink" href="#embedding-projector" title="Link to this heading">¶</a></h3>
<p>When using LIT with a model that returns embeddings (or activations) in addition
to predictions, the embedding projector will show all datapoints by their
embeddings projected down to 3 dimensions. This is useful for exploring and
understanding clusters of datapoints.</p>
<p><a class="reference internal" href="_images/lit-embeddings.png"><img alt="LIT embeddings" class="align-center" src="_images/lit-embeddings.png" style="width: 500px;" /></a></p>
<p>The specific embedding used to generate the projection can be selected in a
dropdown, along with the method of projection (either UMAP or PCA). An
additional drop-down allows changing of the datapoint feature used for the label
of each datapoint. The labels are shown on datapoint hover or click.</p>
<p>The visualization can be rotated through click-and-drag interaction, and panned
through control+click-and-drag. A datapoint can be selected with a click, or a
set of datapoints can be selected using a lasso through a shift+click-and-drag
interaction.</p>
<p>The color of the datapoints is controlled by the color settings in the selection
toolbar.</p>
</section>
<section id="data-table">
<h3>Data Table<a class="headerlink" href="#data-table" title="Link to this heading">¶</a></h3>
<p>The data table shows all datapoints in a simple table. Datapoints can be
selected or unselected through a click. Shift+click allows selecting a set of
consecutive datapoints, and control+click allows selecting a set of individual
datapoints one at a time. Currently selected datapoints are highlighted with a
light blue background. The primary selected datapoint is highlighted with a
darker blue background. If a set of datapoints is currently selected, clicking
on a single datapoint in that set will change it to be the primary selected
datapoint without changing the overall set of selected datapoints.</p>
<p>The default sort order shows datapoints in the order they were loaded from the
dataset, but with newly-generated datapoints being placed directly below their
“source” datapoint, instead of at the end of the table.</p>
<p>The sort order can be changed to sort by columns through use of the up and down
arrows in the table header row. Additionally, the data table can be filtered
through text, regex, numerical ranges, and column-name prefixes using a global
search box. The table can also be filtered by column through a text search using
the search buttons for each column in the header row. All columns that have
filters set on them have their search button outlined. Clicking the <strong>“x”</strong>
button in the search box for a column will clear that column’s filter.</p>
<p>The <strong>“show selected”</strong> checkbox toggles the data table to only show the
datapoints that are currently selected.</p>
<p>The <strong>“show generated”</strong> checkbox toggles the data table to only show generated
datapoints, that is, the datapoints that have been added through modules such as
the <em>Datapoint Editor</em> or the <em>Counterfactual Generators</em>.</p>
<p>The <strong>“reset view”</strong> button returns the data table to its standard, default
view.</p>
<p>A <strong>“columns”</strong> drop-down allows showing/hiding of specific columns to customize
what the data table shows. Model predictions can be added as columns through
this dropdown, but they are not shown in the data table by default, in order to
keep the table decluttered.</p>
<p>Column names that exceed the maximum length are truncated with an ellipsis to
the left, and can be viewed in their entirety when hovered over. Similarly,
table cells that exceed 3 lines of text are truncated with a Show More icon,
which can be clicked to view the full content. Text cells can be collapsed to
their default state using the <strong>“reset view”</strong> button.</p>
<p>The below data table shows one sorted by the “label” field, with the “sentence”
field being filtered to show only those datapoints that contain the word “film”
in them.</p>
<p><a class="reference internal" href="_images/lit-datatable.png"><img alt="LIT data table" class="align-center" src="_images/lit-datatable.png" style="width: 500px;" /></a></p>
<p>A datapoint can be pinned to enable comparison by clicking the pin icon on the
left side of the datapoint’s table entry when the datapoint is hovered over or
selected. A pinned datapoint can be unpinned by clicking on its pin icon again.
Similarly, a datapoint can be starred and unstarred by clicking the neighboring
star icon. Starred datapoints are tracked in an automatically generated Starred
slice for convenience.</p>
<p>You can also export data to CSV using the copy or download buttons in the bottom
right:</p>
<p><a class="reference internal" href="_images/lit-datatable-export.png"><img alt="LIT data table" class="align-center" src="_images/lit-datatable-export.png" style="width: 400px;" /></a></p>
<p>This will export all data in the current table view. To export only the
selection, use the “Show only selected” toggle. To include additional columns
such as model predictions, enable them from the “Columns” dropdown.</p>
</section>
<section id="datapoint-editor">
<h3>Datapoint Editor<a class="headerlink" href="#datapoint-editor" title="Link to this heading">¶</a></h3>
<p>The datapoint editor shows the details of the primary selected datapoint, if one
is selected. Any field can be edited, and a new datapoint created with those
edits through the <strong>“Add”</strong> button. Any edit to an existing datapoint must be
saved as a new datapoint to be explored, to keep datapoints immutable for
simplicity of use.</p>
<p>When no datapoint is selected, the editor shows a blank datapoint that can be
filled out by hand to create a completely new datapoint.</p>
<p>Features shown with a “(*)” next to their name are required as model input and
must be filled out to create a new datapoint. Other fields are optional.</p>
<p><a class="reference internal" href="_images/lit-datapoint-editor.png"><img alt="LIT datapoint editor" class="align-center" src="_images/lit-datapoint-editor.png" style="width: 500px;" /></a></p>
</section>
<section id="datapoint-generator">
<h3>Datapoint Generator<a class="headerlink" href="#datapoint-generator" title="Link to this heading">¶</a></h3>
<p>The datapoint generator module allows creation of new datapoints from all
currently-selected datapoints (or the entire dataset if no datapoints are
selected) through a set of counterfactual datapoint generators. These generators
are provided by the backend and all available generators will show up as buttons
in the module. Clicking one of these buttons causes the creation of new
datapoints that are displayed in a table inside the module and can be added to
the dataset either individually, or altogether, through the add buttons.</p>
<p>Generators built into LIT include:</p>
<ul class="simple">
<li><p><strong>Scrambler</strong>: Scrambles the words in a text feature randomly.</p></li>
<li><p><strong>Back-translation</strong>: Translates a text feature into other languages and
then back to the source language to create paraphrases of the initial text
feature.</p></li>
<li><p><strong>Hotflip</strong>: When analyzing a classification task and the model provides
token-based gradients, this generator will change the token with the highest
influence on the prediction to the token with the most opposite influence.</p></li>
<li><p><strong>Word replacer</strong>: Provides a text box to define a comma-separated set of
replacements to perform (such as “great -&gt; terrible, hi -&gt; hello”).
Counterfactual datapoints are created for any datapoint found that contains
the source word, with it replaced with the provided result word. Word
replacer also supports multiple targets per word with “|” separator. For
example, “great -&gt; terrible | bad” will produce two outputs where “great” is
replaced with “terrible” and “bad”.</p></li>
</ul>
<p>The non-text fields in the generated datapoints can be edited before adding them
to the dataset. This is important in case some datapoint feature is no longer
correct after the counterfactual generation. For example, in a sentiment
classifier, if you used the word replacer generator to replace the word “good”
with “terrible” in the input “this movie is good”, then you probably want to
change the ground truth sentiment of that datapoint from 1 to 0 before you add
it to your dataset for analysis.</p>
<p><img alt="LIT datapoint generator" src="_images/lit-datapoint-generator.png" /></p>
</section>
<section id="metrics-table">
<h3>Metrics Table<a class="headerlink" href="#metrics-table" title="Link to this heading">¶</a></h3>
<p>The metrics table shows model metrics for each model in a table format. The
exact metric types are determined by the python metrics component that
calculates metrics given the model types being evaluated. These can include
measures such as accuracy (for classifiers), error (for regression tasks), and
BLEU score (for translation tasks). By default, the measures are calculated and
shown for the entire dataset, and also for the current selection. Additionally,
through the <strong>“show slices”</strong> checkbox, the metrics table can calculate and
display metrics for each saved slice as well.</p>
<p>There is also a <strong>“Facet by”</strong> set of dataset feature checkboxes; one checkbox
for each feature in the dataset that results can be faceted by. When one or more
of these are checked, the dataset (or current selection, if there is one) is
faceted into sub groups for each of the calculated buckets, and metrics are
displayed for those subsets of the datapoints of interest. This could be used,
for example, to compare metrics for a toxicity classifier, broken down by
gender, assuming the dataset has a categorical gender feature in it.</p>
<p>The below screenshot shows the metrics table with metrics for the entire
dataset, for the dataset faceted into datapoints with label 0 and with label 1,
and also for two named slices that have been created by a user.</p>
<p><img alt="LIT metrics" src="_images/lit-metrics.png" /></p>
</section>
<section id="confusion-matrix">
<h3>Confusion Matrix<a class="headerlink" href="#confusion-matrix" title="Link to this heading">¶</a></h3>
<p>The confusion matrix buckets all datapoints from the dataset (or the current
selection, if one is made) into buckets in a 2D matrix. This is normally used to
compare classification predictions on a model versus the ground truth classes of
the datapoints. In this case, the axes of the matrix are configurable to be set
to any categorical field in the dataset or return from a model. For example,
when comparing two models, the confusion matrix can be set up to show
agreements/disagreements between classifications in the two models, as opposed
to agreements/disagreements between one model’s classifications and the ground
truth.</p>
<p>The individual cells and the row and column headers are all clickable to toggle
on/off selection of the datapoints in that cell or row or column. In this way,
the confusion matrix module can be used to select points of interest, such as
all false positives in a binary classification task, or all datapoints where two
models being compared disagree on classification.</p>
<p><img alt="LIT confusion matrix" src="_images/lit-conf-matrix.png" /></p>
</section>
<section id="scalars">
<h3>Scalars<a class="headerlink" href="#scalars" title="Link to this heading">¶</a></h3>
<p>The scalars module shows a set of scatter or jitter plots, one for each scalar
output of a loaded model (such as a regression score, or a classification score
for a specific class). Each of them contains all datapoints in the dataset, laid
out horizontally by the score. For classification scores, the Y axis is a random
jitter of the data to better view all datapoints. For regression scores, where
ground truth is known, the Y axis is the error in the prediction (points below
the x-axis are under-predicted).</p>
<p>Datapoints can be selected either though clicking, or through lasso selection by
clicking and dragging.</p>
<p>The color of the datapoints is controlled by the color settings in the selection
toolbar.</p>
<p>For binary classification tasks, this module also contains a threshold slider in
order to change the positive classification threshold at which datapoints are
classified as being in the positive class. This slider value defaults to 0.5.</p>
<p>For multi-class classification tasks where a null index (default class) is set
in the model specifications, this module also contains a margin slider for the
non-default classes, to control how high a classification score must be in that
class before a datapoint is classified as that class as opposed to the default
class. The margin value defaults to 0, meaning the class with the highest score
is the class the datapoint is inferred to be.</p>
<p><img alt="LIT prediction scores" src="_images/lit-pred-score.png" /></p>
</section>
<section id="model-output">
<h3>Model Output<a class="headerlink" href="#model-output" title="Link to this heading">¶</a></h3>
<p>Model output modules show the result of a model on the primary selected
datapoint. The visuals of these modules depend on the model task being
performed. For a simple classification task, it will show the class scores from
the model, the predicted class, and, if ground truth is available in the
dataset, it will also show the ground truth classification.</p>
<p><img alt="LIT classification results" src="_images/lit-classification-results.png" /></p>
<p>For structured prediction tasks like span labeling, a span graph module can
display all tagged spans returned by the model, along with a visualization of
the ground truth spans if one is available in the dataset.</p>
<p><a class="reference internal" href="_images/lit-structured-prediction.png"><img alt="LIT structured prediction" class="align-center" src="_images/lit-structured-prediction.png" style="width: 800px;" /></a></p>
</section>
<section id="salience-maps">
<h3>Salience Maps<a class="headerlink" href="#salience-maps" title="Link to this heading">¶</a></h3>
<p>Salience maps show the influence of different parts of inputs features on a
model’s prediction on the primary selection. This module can contain multiple
methodologies for calculating this salience, depending on the capabilities of
the model being analyzed (e.x. if the model provides gradients, then
gradient-based token-wise salience can be calculated and displayed – see
<a class="reference internal" href="api.html#adding-models-and-data"><span class="std std-ref">adding models and data</span></a> for more). The
background of each text piece is colored by the salience of that piece on the
prediction, and hovering on any piece will display the exact value calculated
for that piece.</p>
<p>There is an <strong>“autorun”</strong> button by each methodology on the right side of the
bar (the methodoloy name is on the left side). If it is checked, then that
calculation is made when a new primary datapoint is selected. If it is
unchecked, the calculation isn’t made until it is checked. This can be valuable
so that expensive, long-running saliency calculations (such as LIME) aren’t
performed on every datapoint selection, but only when explicitly asked for.</p>
<p><img alt="LIT saliency maps" src="_images/lit-salience.png" /></p>
</section>
</section>
<section id="user-journeys">
<h2>User Journeys<a class="headerlink" href="#user-journeys" title="Link to this heading">¶</a></h2>
<p>In this section, we explore some example user journeys and how LIT enables them.</p>
<section id="sentiment-analysis">
<h3>Sentiment Analysis<a class="headerlink" href="#sentiment-analysis" title="Link to this heading">¶</a></h3>
<p>How well does a sentiment classifier handle negation? We load the development
set of the Stanford Sentiment Treebank, and use the search function in LIT’s
data table to find the 56 datapoints containing the word “not”. Looking at the
<em>Metrics</em> Table, we find that surprisingly, our BERT model gets 100% of these
correct! But we might want to know if this is truly robust. With LIT, we can
select individual datapoints and look for explanations. For example, take the
negative review, “It’s not the ultimate depression-era gangster movie.”. As
shown in the screenshot below, salience maps suggest that “not” and “ultimate”
are important to the prediction.</p>
<p>We can verify this by creating modified inputs, using LIT’s <em>Datapoint Editor</em>.
Removing “not” gets a strongly positive prediction from “It’s the ultimate
depression-era gangster movie.”, while replacing “ultimate” to get “It’s not the
worst depression-era gangster movie.” elicits a mildly positive score from our
model.</p>
<p><img alt="Sentiment analysis" src="_images/lit-sentiment-analysis.png" /></p>
</section>
<section id="sequence-salience">
<h3>Sequence salience<a class="headerlink" href="#sequence-salience" title="Link to this heading">¶</a></h3>
<p>Sequence salience generalizes token-based salience to text-to-text models,
allowing you to explain the impact of the prompt tokens on parts of the model
output.</p>
<p>Check out <a class="reference internal" href="components.html#sequence-salience"><span class="std std-ref">here</span></a> for more details on how to
navigate the Sequence Salience UI module.</p>
</section>
</section>
</section>

        </article>
      </div>
      <footer>
        
        <div class="related-pages">
          <a class="next-page" href="components.html">
              <div class="page-info">
                <div class="context">
                  <span>Next</span>
                </div>
                <div class="title">Components and Features</div>
              </div>
              <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
            </a>
          <a class="prev-page" href="demos.html">
              <svg class="furo-related-icon"><use href="#svg-arrow-right"></use></svg>
              <div class="page-info">
                <div class="context">
                  <span>Previous</span>
                </div>
                
                <div class="title">Demos</div>
                
              </div>
            </a>
        </div>
        <div class="bottom-of-page">
          <div class="left-details">
            <div class="copyright">
                Copyright &#169; 2023, Google LLC
            </div>
            Made with <a href="https://www.sphinx-doc.org/">Sphinx</a> and <a class="muted-link" href="https://pradyunsg.me">@pradyunsg</a>'s
            
            <a href="https://github.com/pradyunsg/furo">Furo</a>
            
          </div>
          <div class="right-details">
            
          </div>
        </div>
        
      </footer>
    </div>
    <aside class="toc-drawer">
      
      
      <div class="toc-sticky toc-scroll">
        <div class="toc-title-container">
          <span class="toc-title">
            On this page
          </span>
        </div>
        <div class="toc-tree-container">
          <div class="toc-tree">
            <ul>
<li><a class="reference internal" href="#">UI Guide</a><ul>
<li><a class="reference internal" href="#general-layout">General Layout</a><ul>
<li><a class="reference internal" href="#layout-options">Layout Options</a></li>
</ul>
</li>
<li><a class="reference internal" href="#datapoint-selections">Datapoint Selections</a></li>
<li><a class="reference internal" href="#toolbars">Toolbars</a><ul>
<li><a class="reference internal" href="#top-bar">Top Bar</a><ul>
<li><a class="reference internal" href="#global-settings">Global Settings</a></li>
<li><a class="reference internal" href="#url-sharing">URL Sharing</a></li>
</ul>
</li>
<li><a class="reference internal" href="#main-toolbar">Main Toolbar</a></li>
<li><a class="reference internal" href="#status-bar">Status Bar</a></li>
</ul>
</li>
<li><a class="reference internal" href="#comparing-models">Comparing Models</a></li>
<li><a class="reference internal" href="#comparing-datapoints">Comparing Datapoints</a></li>
<li><a class="reference internal" href="#slices">Slices</a></li>
<li><a class="reference internal" href="#module-details">Module Details</a><ul>
<li><a class="reference internal" href="#embedding-projector">Embedding Projector</a></li>
<li><a class="reference internal" href="#data-table">Data Table</a></li>
<li><a class="reference internal" href="#datapoint-editor">Datapoint Editor</a></li>
<li><a class="reference internal" href="#datapoint-generator">Datapoint Generator</a></li>
<li><a class="reference internal" href="#metrics-table">Metrics Table</a></li>
<li><a class="reference internal" href="#confusion-matrix">Confusion Matrix</a></li>
<li><a class="reference internal" href="#scalars">Scalars</a></li>
<li><a class="reference internal" href="#model-output">Model Output</a></li>
<li><a class="reference internal" href="#salience-maps">Salience Maps</a></li>
</ul>
</li>
<li><a class="reference internal" href="#user-journeys">User Journeys</a><ul>
<li><a class="reference internal" href="#sentiment-analysis">Sentiment Analysis</a></li>
<li><a class="reference internal" href="#sequence-salience">Sequence salience</a></li>
</ul>
</li>
</ul>
</li>
</ul>

          </div>
        </div>
      </div>
      
      
    </aside>
  </div>
</div><script src="_static/documentation_options.js?v=f2a433a1"></script>
    <script src="_static/doctools.js?v=9a2dae69"></script>
    <script src="_static/sphinx_highlight.js?v=dc90522c"></script>
    <script src="_static/scripts/furo.js?v=4e2eecee"></script>
    </body>
</html>